#-*- coding:utf-8 -*-
import requests;
from lxml import etree;
import time;
import re;
import mysql.connector;
import uuid;

class LianJia:
	def __init__(self):
		self.header={'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e Safari/602.1'};		
		self.conn=mysql.connector.connect(user='root',password='123456',database='jiang');
		self.cursor=self.conn.cursor();


	#迭代这个小区这一页的数据
	def one_xiaoqu_per_page(self,xiaoquList):
		unit_price_partten=re.compile(r'(\d{4,6})');
		for xiaoqu in xiaoquList:
			data={};
			data['title']=xiaoqu.xpath('div[1]/text()')[0];
			#不统计车位
			if(re.findall(r'车位',data['title'])):continue;
			#单价，单位元
			unit_price=xiaoqu.xpath('div[4]/span[2]/text()')[0];
			result=re.match(unit_price_partten,unit_price);
			if(not result):continue;#有些显示的是：下载app查看成交价，就跳过
			unit_price=result.group(1);			
			data['unit_price']=float(unit_price);
			#总价，单位万
			total_price=xiaoqu.xpath('div[4]/span[1]/em[1]/text()')[0];
			data['total_price']=float(total_price);
			#签约日期
			sign_date=xiaoqu.xpath('div[3]/text()')[0];
			result=re.match(r'签约日期：(.*)',sign_date);
			if(result):
				sign_date=result.group(1);
			data['sign_date']=sign_date;	
			data['xiaoqu_id']=self.xiaoqu_id;
			self.saveDB(data);

	#url是这个小区的成交数据url，这里将迭代这个小区的每一页，最多20页
	def one_xiaoqu(self,url):
		for page in range(1,20):
			time.sleep(1);
			page_url=url+'pg'+str(page);
			print(page_url);
			try:
				html=requests.get(page_url,headers=self.header);
			except:
				self.writeLog('请求url出错：'+page_url);
				continue;	
			selector=etree.HTML(html.text);
			xiaoquList=selector.xpath('//div[@class="item_list"]');
			#判断是否到达未页
			if(len(xiaoquList)<=0):
				break;
			try:	
				self.one_xiaoqu_per_page(xiaoquList);
			except:
				self.writeLog('执行one_xiaoqu_per_page出错，请求url是：'+page_url);
				continue;	
		self.conn.commit();

	def each_xiaoqu(self):
		#小区编号列表文件
		myfile=open('15xiaoqu_id.txt','r');
		#把每一行读出来 存在一个列表里
		lines=myfile.readlines();
		total_num=len(lines);
		i=1;
		for v in lines:
			self.xiaoqu_id=v.strip();
			jd=(i/total_num)*100;
			jd=round(jd,2);
			info='=======正在抓取小区：%s,进度%d/%d,%.2f'%(self.xiaoqu_id,i,total_num,jd)+'% ========';
			print(info);

			#拼接小区成交页面url
			url='https://m.lianjia.com/cd/chengjiao/c%sor1'%self.xiaoqu_id;
			try:
				self.one_xiaoqu(url);
			except:
				self.writeLog('执行one_xiaoqu出错，请求url是：'+url);
				continue;	
			time.sleep(2);
			i=i+1;
			# if(i>3):
			# 	break;

	def saveDB(self,data):
		sql="INSERT INTO lj_deal(title,unit_price,total_price,sign_date,xiaoqu_id) VALUES ('%s',%d,%d,'%s','%s')"\
		%(data['title'],data['unit_price'],data['total_price'],data['sign_date'],data['xiaoqu_id']);
		self.cursor.execute(sql);		

	def writeLog(self,content):
		print(content);
		logfile=open('15log.txt','a');				
		logfile.write(content+'\n');
		logfile.close();

obj=LianJia();
obj.each_xiaoqu();


